#爬取的网站：https://shanghai.anjuke.com/sale/?from=esf_list
import csv # 用于数据存储
import os
import pandas as pd
import requests # 用于请求网页
import chardet # 用于修改编码
import re # 用于提取数据
from lxml import etree # 解析数据的库
import time # 可以粗糙模拟人为请求网页的速度
import warnings # 忽略代码运行时候的警告信息

warnings.filterwarnings("ignore")
def request_url(url):
    cookies = {
        'aQQ_ajkguid': '26FD33D3-F01D-19D8-079C-709C4DD55FB4',
        'ctid': '11',
        'ajk-appVersion': '',
        'id58': 'CrIej2ZZQ0UY78zFFA3tAg==',
        'isp': 'true',
        '58tj_uuid': '1cbf360e-69d3-4609-83f2-b7ab7f02b868',
        'xxzlclientid': '631e09c4-77e6-45fe-a2f3-1717125965332',
        'xxzlxxid': 'pfmxk1nFefldyXqNt7unzt9e5eGRai5E3+rGqauUcxHPPaatkSfuI036TVQHQ0nDzSaY',
        'xxzlbbid': 'pfmbM3wxMDMyMnwxLjkuMXwxNzE3MTI1OTY1NjA0OTUwNTEzfHJma3k5bnJveTJvUFFtMzhKS1JlRDFzVHovS1lpMVVWQ2FCMUR3eGxRT289fGZhNTg0NzllODBhZmQyNDEzMjNlMGQ0ZTI4ODA1NzdmXzE3MTcxMjU5NjQ4MThfY2RiZDYxNmY3ZTYzNGI5Nzg0OGUwMGJhMjgxZTFkMTVfMTAzNDUwNDU1NHw1YzFhYjMxYTMzMGFiNDlkMjczMWFhNWQ2OTI4NjA3N18xNzE3MTI1OTY0NzQ0XzI1Ng==',
        '_ga': 'GA1.2.2028930794.1717750773',
        'new_uv': '2',
        'wmda_uuid': '901790710f54ceffab4784e42ede0070',
        'wmda_new_uuid': '1',
        'wmda_visited_projects': '%3B6289197098934',
        'als': '0',
        '_ga_DYBJHZFBX2': 'GS1.2.1717750772.1.0.1717750772.0.0.0',
        'xxzl_cid': 'd2b9c7cc4f00423c98c9eefacb79c1b3',
        'xxzl_deviceid': 'YwnMb/TT3G8T3OoY9oVO30TAGvpAh5kGz4cTlqnPFmoNempW4vVW5YaHKOsNQ+ft',
        'sessid': '09377B6E-F709-B331-817B-282F8EB37161',
        'twe': '2',
        'fzq_h': '4f1637efddf35e49c70f7e25b4c8c4f6_1718198320056_e55ab26654ba416f996a15d3cc9722d2_1034504554',
        'fzq_js_anjuke_ershoufang_pc': '61b32ca6228f635b275851e2dc22654a_1718198325236_23',
        'obtain_by': '1',
        'xxzl_cid': '385efbcd0f0e483aa2ac1b8773d119b6',
        'xxzl_deviceid': 'SCmJh4fu2Ard2m0pqwjcGDoF333NXhgNaZMw/RFudhOwWjn97Mkjw+moRZXL5KhQ',
    }

    headers = {
        'authority': 'guangzhou.anjuke.com',
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'accept-language': 'zh-CN,zh;q=0.9',
        'cache-control': 'no-cache',
        # Requests sorts cookies= alphabetically
        # 'cookie': 'aQQ_ajkguid=21677498-CE30-45C7-ADD2-8F8E2ABBB414; ctid=12; ajk-appVersion=; seo_source_type=1; id58=CrIej2TkgEu5jmn3DMJVAg==; sessid=36361219-6354-2A42-755A-A002E8A7FD41; twe=2; fzq_h=23fafebd91a95c7d87908a5566cf2b98_1701499005283_187ad6ffd93f4107a658339d9932a17f_2018862970; fzq_js_anjuke_ershoufang_pc=3a3132ef8b005a7f08bbc3b23a901dec_1701499333069_23; obtain_by=2; xxzl_cid=63627e5da0da4c5bbd73b1b11675efe0; xxzl_deviceid=CEcgyp79MmoGL3Ur1OqE8aVW9ce9Xe6TxKD1yCyInA8ggeiP+XR/AfcBRGjoUDej',
        'pragma': 'no-cache',
        'referer': 'https://guangzhou.anjuke.com/sale/tianhe/p6/',
        'sec-ch-ua': '"Chromium";v="118", "Google Chrome";v="118", "Not=A?Brand";v="99"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'document',
        'sec-fetch-mode': 'navigate',
        'sec-fetch-site': 'same-origin',
        'sec-fetch-user': '?1',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
    }

    response = requests.get(url,headers=headers,cookies=cookies)
    html=etree.HTML(response.text)
    return html
def parse_html(area,html):
    result=[]
    # title=''.join(html.xpath('//*[@id="esfMain"]/section/section[3]/section[1]/section[2]/div[1]/a/div[2]/div[1]/div[1]/h3/text()'))
    # model=''.join(html.xpath('//*[@id="esfMain"]/section/section[3]/section[1]/section[2]/div[1]/a/div[2]/div[1]/section/div[1]/p[1]/span/text()')) #可能会报错
    # size=''.join(html.xpath('//*[@id="esfMain"]/section/section[3]/section[1]/section[2]/div[1]/a/div[2]/div[1]/section/div[1]/p[2]/text()'))
    # price=''.join(html.xpath('//*[@id="esfMain"]/section/section[3]/section[1]/section[2]/div[1]/a/div[2]/div[2]/p[1]/span/text()'))
    # time=''.join(html.xpath('//*[@id="esfMain"]/section/section[3]/section[1]/section[2]/div[1]/a/div[2]/div[1]/section/div[1]/p[5]/text()'))
    # m_price=''.join(html.xpath('//*[@id="esfMain"]/section/section[3]/section[1]/section[2]/div[1]/a/div[2]/div[2]/p[2]/text()'))
    # addr=''.join(html.xpath('//*[@id="esfMain"]/section/section[3]/section[1]/section[2]/div[1]/a/div[2]/div[1]/section/div[2]/p[1]/text()'))
    # result.append([title,model,size,price,time,m_price,addr])
    # return result
    '''这里使用的是绝对路径，从而只能获取页面中的第一个元素，所以需要改用相对路径'''
    items = html.xpath('//*[@class="property-content"]')  # 寻找所有的房源列表项
    total_item=len(items)
    for item in items:
        title=''.join(item.xpath('./div/div[@class="property-content-title"]/h3/@title'))
        dic={
        '地区':area,
        '标题':title,
        '地址': ''.join(item.xpath('./div[@class="property-content-detail"]/section/div[2]/p[2]/span/text()')),
        '总价': ''.join(item.xpath('./div[@class="property-price"]/p[1]/span/text()')),
        '单价': ''.join(item.xpath('./div[@class="property-price"]/p[@class="property-price-average"]/text()')).replace('\n','').replace(' ',''),
        '户型': ''.join(item.xpath('./div[@class="property-content-detail"]/section/div[1]/p[1]/span/text()')),
        '面积': ''.join(item.xpath('./div[@class="property-content-detail"]/section/div[1]/p[2]/text()')).replace(' ','').replace('\n',''),
        '朝向': ''.join(item.xpath('./div[@class="property-content-detail"]/section/div[1]/p[3]/text()')).replace(' ', '').replace('\n', ''),
        '楼层': ''.join(item.xpath('./div[@class="property-content-detail"]/section/div[1]/p[4]/text()')).replace(' ', '').replace('\n', ''),
        '年份': ''.join(item.xpath('./div[@class="property-content-detail"]/section/div[1]/p[5]/text()')).replace(' ', '').replace('\n', ''),
        }
        result.append(dic)
    # print(result[0])
    return result



def save_to_csv(result,path):
    file_exists = os.path.isfile(path)
    with open(path,'a',newline='',encoding='utf-8') as f:
        writer = csv.DictWriter(f,fieldnames=result[0].keys())
        if not file_exists:
            writer.writeheader()
        for row in result:
            writer.writerow(row)

def main():
    start_time = time.time()  # 记录开始时间
    areas={ #不同地区
        'pudong':'浦东',
        'minhang':'闵行',
        'songjiang':'松江',
        'baoshan':'宝山',
        # 'jiading':'嘉定',
        # 'xuhui':'徐汇',
        # 'qingpu':'青浦',
        # 'jingan':'静安',
        # 'putuo':'普陀',
        # 'yangpu':'杨浦',
        # 'fengxian':'奉贤',
        # 'huangpu':'黄浦',
        # 'hongkou':'虹口',
        # 'changning':'长宁',
        # 'jinshan':'金山',
        # 'chongming':'崇明',
        # 'shanghaizhoubian':'上海周边'
    }
    for key,area in areas.items():
        for page in range(1,4):
            url='https://shanghai.anjuke.com/sale/'+key+'/b10-p'+str(page)+'/?from=esf_list'
            print('开始爬取',url)
            html=request_url(url)
            data=parse_html(area,html)
            save_to_csv(data,'shanghai.csv')
            time.sleep(5)
    end_time = time.time()  # 记录结束时间
    elapsed_time = end_time - start_time  # 计算爬取耗时

    print(f'爬取完成！耗时: {elapsed_time:.2f} 秒')

if __name__ == '__main__':
    main()